import pymongo
from selenium import webdriver
import time
import random

def connect_mongo(database_name,sheet_name):
    client = pymongo.MongoClient('localhost',27017)
    database = client[database_name]
    sheet_name = database[sheet_name]
    return sheet_name


#从页面提取股票代码
def browser_code(browser,result):
    codes = browser.find_elements_by_xpath('//*[@id="tbl_wrap"]/table/tbody/tr/th[1]/a')
    for code in codes:
        result.append(code.text)

#读取数据并翻页
def industry_detail(url):
    browser = webdriver.Chrome()
    browser.get(url)
    results = []
    time.sleep(2)
    browser_code(browser,results)
    print('page 1')
    try:
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser,results)
        print('page 2')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 3')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 4')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 5')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 6')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 7')
        next_page = browser.find_element_by_xpath('//*[@id="list_page_btn_2"]/a[1]')
        next_page.click()
        time.sleep(2)
        browser_code(browser, results)
        print('page 8')
        browser.close()
    except:
        print('Last Page!')
        browser.close()
        return results


def get_industy_link(url):
    browser = webdriver.Chrome()
    browser.get(url)
    name_paths = browser.find_elements_by_xpath('//*[@id="datatbl"]/tbody/tr/td[1]/a')
    nums = browser.find_elements_by_xpath('//*[@id="datatbl"]/tbody/tr/td[2]')
    for name_path,num in zip(name_paths,nums):
        name = name_path.text
        link = name_path.get_attribute('href')
        comp_num = num.text
        insert_data = {
            'indust_name':name,
            'parse_link':link,
            'comp_num':comp_num
        }
        print(insert_data)
        connect_mongo('stock','indust_name_link').insert_one(insert_data)
    browser.close()



if __name__ == '__main__':
    contents = connect_mongo('stock', 'indust_name_link').find()
    for out_data in contents:
        time.sleep(random.uniform(1,2))
        indust_name = out_data['indust_name']
        parse_link = out_data['parse_link']
        comp_num = out_data['comp_num']
        parse_codes = industry_detail(parse_link)
        for code in parse_codes:
            if code == '':
                insert_data = {
                    'code': code
                }
                connect_mongo('industry', indust_name).insert_one(insert_data)
        if len(parse_codes) != int(comp_num):
            print('{0} is Error!'.format(indust_name))



